//============================================================================
// Name        : ManagerBerthier.cpp
// Author      : 
// Version     :
// Copyright   : Your copyright notice
// Description : Hello World in C++, Ansi-style
//============================================================================

#include <iostream>
#include "ManagerBerthier.h"
#include <ext/hash_map>
#include <list>
#include <vector>
#include <queue>
#include "../classes_net/URLFileDealer.h"
#include "../classes_net/DownloadParser.h"
#include "Hasher.h"
#include<string>
#include <boost/tr1/memory.hpp>
#include <boost/thread/mutex.hpp>
#include "../classes_net/URL.h"
#include "../classes_outros/DownloaderQueueException.h"
#include "../classes_outros/Config.h"
#include "../classes_outros/RobotsDownloadInterface.h"
#include "ThreadFunctionDatabase.h"

using namespace std;
using namespace __gnu_cxx;

std::tr1::shared_ptr<ManagerBerthier> ManagerBerthier::theManagerBerthier(new ManagerBerthier());
const unsigned int ManagerBerthier::URLS_PER_QUEUE = Config::getUrlsPerQueue();

ManagerBerthier::ManagerBerthier() 
:timeBetweenVisits(Config::getTimeBetweenVisits()),
 queuesInitialized(false)
{
}



void ManagerBerthier::initializeDownloaderQueues (const int quantity) throw(DownloaderQueueException){
	if (queuesInitialized) {
		DownloaderQueueException dqe("As filas jah foram inicializadas");
		throw dqe;
	}
	urlQueueVector.reserve(quantity);
	for (int i = 0; i<quantity; i++)
		urlQueueVector.push_back(queue<URL *>() );	
	cout << "mb::init TAMANHO urlQueueVector " << urlQueueVector.size() << endl; 
	downloaderQueueMutex = new boost::mutex[quantity];
	numQueues = quantity;
	queuesInitialized = true;
}

bool ManagerBerthier::refillQueuesOnce() throw(DownloaderQueueException){
	if (!queuesInitialized){
		DownloaderQueueException dqe("MB::refillQueuesOnce chamado antes de inicializar vetor");
		throw dqe;
	}
	
	bool conseguiu = false;
	
	{
//	cout << "Lock: 000 A" << endl;
	boost::mutex::scoped_lock urlQueueLock(urlQueueBufferMutex);
//	cout << "Lock: 000 B" << endl;
	if (urlQueueBuffer.size() < numQueues)
		refillQueueBuffer();
	}
	vector< queue<URL *> >::iterator it;

	
	int i = 0;
	URL * url = NULL;
	bool hasNewUrl = false;
	for (it = urlQueueVector.begin(); (it!=urlQueueVector.end()) && !urlQueueBuffer.empty(); it++) {
		if (!hasNewUrl)
			url = findURL();
//		cout << "Lock: 001 A" << endl;
		boost::mutex::scoped_lock downloaderQueueLock(downloaderQueueMutex[i++]);
//		cout << "Lock: 001 B" << endl;
		if (url == NULL) {
			cout << "\n\n\n\n ********************NULL********************\n\n\n\n" << endl;
			return false;
		}
		if (it->size() <= URLS_PER_QUEUE){
			it->push(url);
			hasNewUrl = false;
			conseguiu = true;
		}
		else {
			hasNewUrl = true;
		}
	}
	return conseguiu;
}

URL * ManagerBerthier::getNewURL(int id) throw(DownloaderQueueException){
	
	if (!queuesInitialized){
		DownloaderQueueException dqe ("As filas dos downloaders n�o foram inicializadas");
		throw dqe;
	}
	
	URL * url;
//	cout << "Lock: 002 A" << endl;
	boost::mutex::scoped_lock downloaderQueueLock(downloaderQueueMutex[id]);
//	cout << "Lock: 002 B" << endl;
	if (urlQueueVector[id].empty()) {
		return NULL;
	}
	else {
//		cout << "mb::getNew-Tirando elemento da fila " << id << ", de tamanho " << urlQueueVector[id].size() << endl;
		url = urlQueueVector[id].front();
		urlQueueVector[id].pop();
		return url;
	}	
}


string ManagerBerthier::getRobots(string domain) {
	DownloadParser downloader;
//	cout << "Puxando o robots.txt do dominio " << domain << endl;
	downloader.setURL(domain);
	return downloader.getRobot();
}

/**
 * Verificar se a url � v�lida qdo obter!!!
 */
URL * ManagerBerthier::findURL() {
	URL * url;
	time_t timeMc;
	std::tr1::shared_ptr<URLFileDealer> urlFileDealer = URLFileDealer::getInstance();
	//Ele bloqueia na inicializacao e desbloqueia no destrutor (nao preciso me preocupar)
	while (true) {
		{
//			cout << "Lock: 003 A" << endl;
			boost::mutex::scoped_lock urlQueueLock(urlQueueBufferMutex);
//			cout << "Lock: 003 B" << endl;
			if (urlQueueBuffer.empty()) {

				bool conseguiu = refillQueueBuffer();

				if (!conseguiu) {
					//cout << "mb:find-RETORNANDO NULL" << endl;
					return NULL; 
				}
			}

			
			url = urlQueueBuffer.front(); 
			urlQueueBuffer.pop();	

		}
		
		
//		unsigned long long domainHash = Hasher::hash64FromString( url->get_dominio() );
		string domainHash = Hasher::md5( url->get_dominio() );

		
		//Se o dom�nio n�o existir, insiro no hash de dominios, e retorno a url
//		cout << "Lock: 004 A" << endl;
		boost::mutex::scoped_lock domainHashMapLock(domainHashMapMutex);
//		cout << "Lock: 004 B" << endl;
		if(domainHashMap[domainHash] == NULL){
			time(&timeMc);
			Dominio * dominio = new Dominio(url->get_dominio(), /*robots,*/ timeMc - timeBetweenVisits);

			if (!RobotsDownloadInterface::put(dominio)) {
				delete dominio;
				urlFileDealer->addToDiskBuffer(url);
		    	domainHashMapLock.unlock();
				continue;
			}
			
			domainHashMap[domainHash] = dominio;
	    	domainHashMapLock.unlock();

			urlFileDealer->addToDiskBuffer(url);
		}
		//se o dominio j� existir, verifico o tempo e permiss�o do robots
		else {
			time(&timeMc);
			Dominio * dominio = domainHashMap[domainHash];
			
			domainHashMapLock.unlock();
			if ( !dominio->isRobotsValid() ) {
				urlFileDealer->addToDiskBuffer(url);
				continue;
			}

			if (!dominio->isAllowedByRobots(url)) {
//				cout << "Robots rejeitou: " << url->toString() << endl; 
				delete url;
				continue;
			}
			else if ( ( timeMc - dominio->getTimestamp() ) < timeBetweenVisits) {
				urlFileDealer->addToDiskBuffer(url);
				continue;
			}
			else {
				dominio->updateTimestamp(timeMc);
				return url;
			}
		}
	}
}




bool ManagerBerthier::refillQueueBuffer() {
	std::tr1::shared_ptr<URLFileDealer> theURLFileDealer = URLFileDealer::getInstance();
	list<URL *> urlList = theURLFileDealer->getNewURLs();
	if (urlList.size() == 0)
		return false;
	list<URL *>::const_iterator it;
	
	for (it = urlList.begin(); it!=urlList.end(); it++) {
		urlQueueBuffer.push(*it);
	}
//	cout << "MC tb suga pelo menos " << urlQueue.size() << " pessoa por dia" << endl;
	return true;	
}